import org.apache.spark.{SparkConf, SparkContext}

/**
  * Spark 方式将 apache 日志进行 计算操作
  */
object SparkETLDemo {
  def main(args: Array[String]){
     System.setProperty("hadoop.home.dir","D:\\hadoop\\hadoop-2.7.3")
     System.setProperty("user.name","hfds")
     System.setProperty("HADOOP_USER_NAME","hdfs")
     if(args.length <1){
        System.err.println("Usage: file ")
        System.exit(1)
     }
    // Spark 程序初始化类
    val conf = new SparkConf().setMaster("local").setAppName("test")
    val sc = new SparkContext(conf)
    var rdd = sc.textFile(args(0))
    // 写入数据到hdfs 系统
    rdd.repartition(1).saveAsTextFile(args(1))
    sc.stop()
  }
}
